from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import matplotlib

def computeNumOfDocsAtEachIndexKeptLevelForXdocBusiness():
    print "computeNumOfDocsAtEachIndexKeptLevelForXdocBusiness(...) called."
    # for gov2
    # numOfPostingsInTotal =  6451948010
    # for clueweb09B
    numOfPostingsInTotal = 17075485964
    
    numOfPostingsAt1Percent = int(numOfPostingsInTotal * 0.01)
    numOfPostingsAt2Percent = int(numOfPostingsInTotal * 0.02)
    numOfPostingsAt3Percent = int(numOfPostingsInTotal * 0.03)
    numOfPostingsAt4Percent = int(numOfPostingsInTotal * 0.04)
    numOfPostingsAt5Percent = int(numOfPostingsInTotal * 0.05)
    numOfPostingsAt6Percent = int(numOfPostingsInTotal * 0.06)
    numOfPostingsAt7Percent = int(numOfPostingsInTotal * 0.07)
    numOfPostingsAt8Percent = int(numOfPostingsInTotal * 0.08)
    numOfPostingsAt9Percent = int(numOfPostingsInTotal * 0.09)
    numOfPostingsAt10Percent = int(numOfPostingsInTotal * 0.1)
    numOfPostingsAt15Percent = int(numOfPostingsInTotal * 0.15)
    numOfPostingsAt20Percent = int(numOfPostingsInTotal * 0.2)
    numOfPostingsAt30Percent = int(numOfPostingsInTotal * 0.3)
    numOfPostingsAt40Percent = int(numOfPostingsInTotal * 0.4)
    numOfPostingsAt50Percent = int(numOfPostingsInTotal * 0.5)
    numOfPostingsAt60Percent = int(numOfPostingsInTotal * 0.6)
    numOfPostingsAt70Percent = int(numOfPostingsInTotal * 0.7)
    numOfPostingsAt80Percent = int(numOfPostingsInTotal * 0.8)
    numOfPostingsAt90Percent = int(numOfPostingsInTotal * 0.9)
    numOfPostingsAt100Percent = int(numOfPostingsInTotal * 1.0)
    numOfPostingsAt1PercentTag = True
    numOfPostingsAt2PercentTag = True
    numOfPostingsAt3PercentTag = True
    numOfPostingsAt4PercentTag = True
    numOfPostingsAt5PercentTag = True
    numOfPostingsAt6PercentTag = True
    numOfPostingsAt7PercentTag = True
    numOfPostingsAt8PercentTag = True
    numOfPostingsAt9PercentTag = True
    numOfPostingsAt10PercentTag = True
    numOfPostingsAt15PercentTag = True
    numOfPostingsAt20PercentTag = True
    numOfPostingsAt30PercentTag = True
    numOfPostingsAt40PercentTag = True
    numOfPostingsAt50PercentTag = True
    numOfPostingsAt60PercentTag = True
    numOfPostingsAt70PercentTag = True
    numOfPostingsAt80PercentTag = True
    numOfPostingsAt90PercentTag = True
    numOfPostingsAt100PercentTag = True
    
    # for clueweb09B
    # sorted by # of postings
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_clueweb09B_20140611_sortedByNumOfPostings"
    # sorted by xDoc
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_clueweb09B_20140611_sortedByXdoc"
    # sorted by Xdoc * log(# of postings)
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocTimesLogNumOfPostings"
    # sorted by Xdoc / NumOfPostings
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocDividedByNumOfPostings"
    # sorted by Xdoc / SQRT(NumOfPostings)
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocDividedBySQRTNumOfPostings"
    # sorted By Xdoc * SQRT(NumOfPostings)
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocTimesSQRTNumOfPostings"
    # sorted By Xdoc / Log(NumOfPostings)
    # inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocDividedByLogNumOfPostings"
    # sorted By Xdoc * Log(NumOfPostings)
    inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByXdocTimesLogNumOfPostings"
    # for gov2
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocDividedByLogNum_20150505"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocMultipleByLogNum_20150505"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocSquareDividedByNumOfPostings_20140601"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByNumOfPostings_20150519"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocValues_20140602"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocDividedByLogNum_20150505"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sortedByXdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument"
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_sortedByXdocDividedBySqrtNum_20150506"
    print "inputFileName:",inputFileName
    inputFileHandler = open(inputFileName,"r")
    numOfDocumentsCanBeIncluded = 0
    numOfPostingsCurrent = 0
    
    print "%,numOfDocumentsCanBeIncluded,numOfPostingsCurrent,numOfPostingsAt1Percent"
    currentLine = inputFileHandler.readline()
    numOfDocumentsCanBeIncluded += 1
    while currentLine:
        # for clueweb09B
        numOfPostingsCurrent += int(currentLine.strip().split(" ")[1])
        # for gov2 
        # numOfPostingsCurrent += int(currentLine.strip().split(" ")[2])
    
        if numOfPostingsCurrent >= numOfPostingsAt1Percent and numOfPostingsAt1PercentTag:
            
            print "1%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt1Percent
            numOfPostingsAt1PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt2Percent and numOfPostingsAt2PercentTag:
            
            print "2%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt2Percent
            numOfPostingsAt2PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt3Percent and numOfPostingsAt3PercentTag:
            
            print "3%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt3Percent
            numOfPostingsAt3PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt4Percent and numOfPostingsAt4PercentTag:
            
            print "4%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt4Percent
            numOfPostingsAt4PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt5Percent and numOfPostingsAt5PercentTag:
            
            print "5%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt5Percent
            numOfPostingsAt5PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt6Percent and numOfPostingsAt6PercentTag:
            print "6%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt6Percent
            numOfPostingsAt6PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt7Percent and numOfPostingsAt7PercentTag:
            print "7%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt7Percent
            numOfPostingsAt7PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt8Percent and numOfPostingsAt8PercentTag:
            print "8%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt8Percent
            numOfPostingsAt8PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt9Percent and numOfPostingsAt9PercentTag:
            print "9%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt9Percent
            numOfPostingsAt9PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt10Percent and numOfPostingsAt10PercentTag:
            print "10%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt10Percent
            numOfPostingsAt10PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt15Percent and numOfPostingsAt15PercentTag:
            print "15%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt15Percent
            numOfPostingsAt15PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt20Percent and numOfPostingsAt20PercentTag:
            print "20%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt20Percent
            numOfPostingsAt20PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt30Percent and numOfPostingsAt30PercentTag:
            print "30%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt30Percent
            numOfPostingsAt30PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt40Percent and numOfPostingsAt40PercentTag:
            print "40%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt40Percent
            numOfPostingsAt40PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt50Percent and numOfPostingsAt50PercentTag:
            print "50%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt50Percent
            numOfPostingsAt50PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt60Percent and numOfPostingsAt60PercentTag:
            print "60%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt60Percent
            numOfPostingsAt60PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt70Percent and numOfPostingsAt70PercentTag:
            print "70%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt70Percent
            numOfPostingsAt70PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt80Percent and numOfPostingsAt80PercentTag:
            print "80%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt80Percent
            numOfPostingsAt80PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt90Percent and numOfPostingsAt90PercentTag:
            print "90%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt90Percent
            numOfPostingsAt90PercentTag = False
    
        if numOfPostingsCurrent >= numOfPostingsAt100Percent and numOfPostingsAt100PercentTag:
            print "100%",numOfDocumentsCanBeIncluded-1,numOfPostingsCurrent,numOfPostingsAt100Percent
            numOfPostingsAt100PercentTag = False
        
        currentLine = inputFileHandler.readline()
        numOfDocumentsCanBeIncluded += 1
    inputFileHandler.close()
    print "Job DONE."

def computeXdocValuesForClueweb09BDocuments(beginningDocID,beginningOffset,endingDocID,outputFileHandler,inputFileHandler,fileSize,termIDANDProbablityDict):
    totalNumOfPostings = 0
    totalNumOfDocs = 0
    inputFileHandler.seek(beginningOffset)
    numOfBytes = 0
    while numOfBytes < fileSize:
            byteString = inputFileHandler.read(4 + 4)
            (docID,numOfPostings) = unpack( "2I", byteString)
            # print docID,numOfPostings
            
            currentXDocValue = 0.0
            for i in range(0,numOfPostings):
                byteString = inputFileHandler.read(4)
                (currentTermID,) = unpack( "1I", byteString)
                currentTermIDProbablity = 0.0
                currentTermIDInStringFormat = str(currentTermID)
                if currentTermIDInStringFormat in termIDANDProbablityDict:     
                    currentTermIDProbablity = termIDANDProbablityDict[currentTermIDInStringFormat]
                else:
                    currentTermIDProbablity = termIDANDProbablityDict["UNK"]
                # print i,currentTermID,currentTermIDProbablity
                currentXDocValue += currentTermIDProbablity
            # print docID,numOfPostings,currentXDocValue
            outputFileHandler.write(str(docID) + " " + str(numOfPostings) + " " + str(currentXDocValue) + "\n")
            
            totalNumOfPostings += numOfPostings
            totalNumOfDocs += 1
            numOfBytes += 4 + 4 + numOfPostings * 4
            # inputFileHandler.seek(numOfBytes)

            if docID == (endingDocID-1):
                break
            # exit(1)
            
            if totalNumOfDocs % 10000 == 0:
                print totalNumOfDocs,"processed."
            
    print "totalNumOfPostings:",totalNumOfPostings
    print "totalNumOfDocs:",totalNumOfDocs

def computeTheVariationsOfTheCombinationOfTheXdocANDNumOfPostings():
    # for gov2
    # inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument_sortedByXdoc"
    # outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_Xdocs_and_variations_20140505Night"
    # for clueweb09B
    inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_clueweb09B_20140611_sortedByDocID"
    outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/xDocValues_variations_clueweb09B_20140611_sortedByDocID"
    inputFileHandler = open(inputFileName,"r")
    outputFileHandler = open(outputFileName,"w")
    
    currentLine = inputFileHandler.readline()
    lineCounter = 1
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        # for gov2
        # currentNumOfPostings = int(currentLineElements[2])
        # currentXdocValue = float(currentLineElements[7])
        # for clueweb09B
        currentDocID = int(currentLineElements[0])
        currentNumOfPostings = int(currentLineElements[1])
        currentXdocValue = float(currentLineElements[2])
        # print currentDocID,currentNumOfPostings,currentXdocValue
        if currentNumOfPostings == 0 or currentXdocValue == 0.0:
            currentSqureRootNumOfPostings = 0.0
            currentLogNumOfPostings = 0.0
            currentNewComputeValue0 = 0.0
            currentNewComputeValue1 = 0.0
            currentNewComputeValue2 = 0.0
            currentNewComputeValue3 = 0.0
            currentNewComputeValue4 = 0.0
        else:
            currentSqureRootNumOfPostings = math.sqrt(currentNumOfPostings)
            currentLogNumOfPostings = math.log(currentNumOfPostings,10)
            currentNewComputeValue0 = currentXdocValue / currentNumOfPostings
            currentNewComputeValue1 = currentXdocValue / currentSqureRootNumOfPostings
            currentNewComputeValue2 = currentXdocValue * currentSqureRootNumOfPostings
            if currentLogNumOfPostings == 0.0:
                currentNewComputeValue3 = 0.0
                currentNewComputeValue4 = 0.0
            else:
                currentNewComputeValue3 = currentXdocValue / currentLogNumOfPostings
                currentNewComputeValue4 = currentXdocValue * currentLogNumOfPostings
        # for clueweb09B format
        # column0: docID
        # column1: # of unique terms
        # column2: Xdoc
        # column3: Xdoc / (# of unique terms)
        # column4: Xdoc / sqrt(# of unique terms)
        # column5: Xdoc * sqrt(# of unique terms)
        # column6: Xdoc / log(# of unique terms)
        # column7: Xdoc * log(# of unique terms)
        outputLine = str(currentDocID) + " " + str(currentNumOfPostings) + " " + str(currentXdocValue) + " " + str(currentNewComputeValue0) + " " + str(currentNewComputeValue1) + " " + str(currentNewComputeValue2) + " " + str(currentNewComputeValue3) + " " + str(currentNewComputeValue4) + "\n" 
        outputFileHandler.write(outputLine)
        if lineCounter % 1000000 == 0:
            print lineCounter,"lines processed."
        currentLine = inputFileHandler.readline()
        lineCounter += 1
    outputFileHandler.close()
    inputFileHandler.close()
    
    print "OVERALL:"
    print "inputFileName:",inputFileName
    print "outputFileName:",outputFileName
    exit(1)

print "Begins..."
computeNumOfDocsAtEachIndexKeptLevelForXdocBusiness()
exit(1)

computeTheVariationsOfTheCombinationOfTheXdocANDNumOfPostings()
exit(1)


# pre load the probabilities
# for clueweb09B, unknown probability string
termIDANDProbablityDict = {}
unigramModelInputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/LMs/Clueweb09B/wholeLexicon_Clueweb09B_goodTuring_unigramProbablity_20140610_SIMPLIFIED"
unigramModelInputFileHandler = open(unigramModelInputFileName,"r")
currentLine = unigramModelInputFileHandler.readline()
lineCounter = 1
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTermProbablity = float(currentLineElements[1])
    if currentTermID not in termIDANDProbablityDict:
        termIDANDProbablityDict[currentTermID] = currentTermProbablity
    currentLine = unigramModelInputFileHandler.readline()
    lineCounter += 1
print "len(termIDANDProbablityDict):",len(termIDANDProbablityDict)
print "termIDANDProbablityDict['UNK']:",termIDANDProbablityDict['UNK']
print "termIDANDProbablityDict['0']:",termIDANDProbablityDict['0']

if len(sys.argv) != 5:
    print "USAGE: programName beginningDocID beginningOffset endingDocID outputFileName"
    exit(1)

beginningDocID = int(sys.argv[1])
beginningOffset = int(sys.argv[2])
endingDocID = int(sys.argv[3])
outputFileName = str(sys.argv[4])
print 'beginningDocID:', beginningDocID
print 'beginningOffset:',beginningOffset
print 'endingDocID:', endingDocID
print 'outputFileName:', outputFileName
outputFileHandler = open(outputFileName,"w")

inputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09B_simple_document_posting_array_20140606"
inputFileHandler = open(inputFileName,"rb")
statinfo = os.stat(inputFileName)
fileSize = statinfo.st_size
print "file size:",fileSize

computeXdocValuesForClueweb09BDocuments(beginningDocID,beginningOffset,endingDocID,outputFileHandler,inputFileHandler,fileSize,termIDANDProbablityDict)
inputFileHandler.close()
outputFileHandler.close()
print 'outputFileName:', outputFileName
print "Ends."

'''
# load the probabilities
# for clueweb09B, unknown probability string
unKnownProbabilityString = "6.33876e-10"
termIDANDProbablityDict = {}
termIDANDProbablityDict["UNK"] = float(unKnownProbabilityString)
unigramModelInputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/LMs/Clueweb09B/wholeLexicon_Clueweb09B_goodTuring_unigramProbablity_20140610"
unigramModelInputFileHandler = open(unigramModelInputFileName,"r")
currentLine = unigramModelInputFileHandler.readline()
lineCounter = 1
while currentLine:
    currentLineElements = currentLine.strip().split(" ")
    if currentLineElements[1] != unKnownProbabilityString:
        currentTermID = currentLineElements[0]
        currentTermProbablity = float(currentLineElements[1])
        if currentTermID not in termIDANDProbablityDict:
            termIDANDProbablityDict[currentTermID] = currentTermProbablity
    else:
        pass
    if lineCounter % 1000000 == 0:
        print lineCounter,"processed."
    currentLine = unigramModelInputFileHandler.readline()
    lineCounter += 1
print "len(termIDANDProbablityDict):",len(termIDANDProbablityDict)
print "termIDANDProbablityDict['UNK']:",termIDANDProbablityDict['UNK']
print "termIDANDProbablityDict['0']:",termIDANDProbablityDict['0']

# temp DUMP
outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/data/LMs/Clueweb09B/wholeLexicon_Clueweb09B_goodTuring_unigramProbablity_20140610_SIMPLIFIED_RAW"
outputFileHandler = open(outputFileName,"w")
for termID in termIDANDProbablityDict:
    outputFileHandler.write(str(termID) + " " + str(termIDANDProbablityDict[termID]) + "\n")
outputFileHandler.close()
print "Overall:"
print "unigramModelInputFileName:",unigramModelInputFileName
print "outputFileName:",outputFileName
'''




